{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Format DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(506, 14)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CRIM</th>\n",
       "      <th>ZN</th>\n",
       "      <th>INDUS</th>\n",
       "      <th>CHAS</th>\n",
       "      <th>NOX</th>\n",
       "      <th>RM</th>\n",
       "      <th>AGE</th>\n",
       "      <th>DIS</th>\n",
       "      <th>RAD</th>\n",
       "      <th>TAX</th>\n",
       "      <th>PTRATIO</th>\n",
       "      <th>B</th>\n",
       "      <th>LSTAT</th>\n",
       "      <th>median_value</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.00632</td>\n",
       "      <td>18.0</td>\n",
       "      <td>2.31</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.538</td>\n",
       "      <td>6.575</td>\n",
       "      <td>65.2</td>\n",
       "      <td>4.0900</td>\n",
       "      <td>1.0</td>\n",
       "      <td>296.0</td>\n",
       "      <td>15.3</td>\n",
       "      <td>396.90</td>\n",
       "      <td>4.98</td>\n",
       "      <td>24.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.02731</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.07</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.469</td>\n",
       "      <td>6.421</td>\n",
       "      <td>78.9</td>\n",
       "      <td>4.9671</td>\n",
       "      <td>2.0</td>\n",
       "      <td>242.0</td>\n",
       "      <td>17.8</td>\n",
       "      <td>396.90</td>\n",
       "      <td>9.14</td>\n",
       "      <td>21.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.02729</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.07</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.469</td>\n",
       "      <td>7.185</td>\n",
       "      <td>61.1</td>\n",
       "      <td>4.9671</td>\n",
       "      <td>2.0</td>\n",
       "      <td>242.0</td>\n",
       "      <td>17.8</td>\n",
       "      <td>392.83</td>\n",
       "      <td>4.03</td>\n",
       "      <td>34.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.03237</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.18</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.458</td>\n",
       "      <td>6.998</td>\n",
       "      <td>45.8</td>\n",
       "      <td>6.0622</td>\n",
       "      <td>3.0</td>\n",
       "      <td>222.0</td>\n",
       "      <td>18.7</td>\n",
       "      <td>394.63</td>\n",
       "      <td>2.94</td>\n",
       "      <td>33.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.06905</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.18</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.458</td>\n",
       "      <td>7.147</td>\n",
       "      <td>54.2</td>\n",
       "      <td>6.0622</td>\n",
       "      <td>3.0</td>\n",
       "      <td>222.0</td>\n",
       "      <td>18.7</td>\n",
       "      <td>396.90</td>\n",
       "      <td>5.33</td>\n",
       "      <td>36.2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      CRIM    ZN  INDUS  CHAS    NOX     RM   AGE     DIS  RAD    TAX  \\\n",
       "0  0.00632  18.0   2.31   0.0  0.538  6.575  65.2  4.0900  1.0  296.0   \n",
       "1  0.02731   0.0   7.07   0.0  0.469  6.421  78.9  4.9671  2.0  242.0   \n",
       "2  0.02729   0.0   7.07   0.0  0.469  7.185  61.1  4.9671  2.0  242.0   \n",
       "3  0.03237   0.0   2.18   0.0  0.458  6.998  45.8  6.0622  3.0  222.0   \n",
       "4  0.06905   0.0   2.18   0.0  0.458  7.147  54.2  6.0622  3.0  222.0   \n",
       "\n",
       "   PTRATIO       B  LSTAT  median_value  \n",
       "0     15.3  396.90   4.98          24.0  \n",
       "1     17.8  396.90   9.14          21.6  \n",
       "2     17.8  392.83   4.03          34.7  \n",
       "3     18.7  394.63   2.94          33.4  \n",
       "4     18.7  396.90   5.33          36.2  "
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from sklearn.datasets import load_boston\n",
    "\n",
    "data = load_boston()\n",
    "train_df = pd.DataFrame(data=data.data, columns=data.feature_names)\n",
    "train_df[\"median_value\"] = data.target\n",
    "\n",
    "print(train_df.shape)\n",
    "train_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Set Up Environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cross-Experiment Key:   '2YztSDxi5paDQaugUYdWoxknmDUq4I-UBXQHCzXUIyg='\n"
     ]
    }
   ],
   "source": [
    "from hyperparameter_hunter import Environment, CVExperiment\n",
    "from sklearn.metrics import r2_score\n",
    "from sklearn.model_selection import RepeatedKFold\n",
    "\n",
    "env = Environment(\n",
    "    train_dataset=train_df,\n",
    "    results_path=\"HyperparameterHunterAssets\",\n",
    "    target_column=\"median_value\",\n",
    "    metrics=dict(r2=r2_score),\n",
    "    cv_type=RepeatedKFold,\n",
    "    cv_params=dict(n_repeats=2, n_splits=5, random_state=42),\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now that HyperparameterHunter has an active `Environment`, we can do two things:\n",
    "\n",
    "# 1. Perform Experiments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<15:23:45> Validated Environment:  '2YztSDxi5paDQaugUYdWoxknmDUq4I-UBXQHCzXUIyg='\n",
      "<15:23:45> Initialized Experiment: '3b699656-e442-4f21-afa8-80099e81427f'\n",
      "<15:23:45> Hyperparameter Key:     'Bl2Yb6WpmWUnmspQM0IGGkgIix_b2EsMI0V71x8RkUQ='\n",
      "<15:23:45> \n",
      "<15:23:45> \n",
      "<15:23:45> F0.0 AVG:   OOF(r2=0.88616)  |  Time Elapsed: 0.05149 s\n",
      "<15:23:45> F0.1 AVG:   OOF(r2=0.87997)  |  Time Elapsed: 0.04793 s\n",
      "<15:23:45> F0.2 AVG:   OOF(r2=0.83099)  |  Time Elapsed: 0.04655 s\n",
      "<15:23:45> F0.3 AVG:   OOF(r2=0.91790)  |  Time Elapsed: 0.04865 s\n",
      "<15:23:45> F0.4 AVG:   OOF(r2=0.87084)  |  Time Elapsed: 0.04752 s\n",
      "<15:23:45> Repetition 0 AVG:   OOF(r2=0.87919)  |  Time Elapsed: 0.24743 s\n",
      "<15:23:45> \n",
      "<15:23:45> F1.0 AVG:   OOF(r2=0.89532)  |  Time Elapsed: 0.04824 s\n",
      "<15:23:45> F1.1 AVG:   OOF(r2=0.84094)  |  Time Elapsed: 0.04853 s\n",
      "<15:23:45> F1.2 AVG:   OOF(r2=0.81587)  |  Time Elapsed: 0.04729 s\n",
      "<15:23:45> F1.3 AVG:   OOF(r2=0.86505)  |  Time Elapsed: 0.04734 s\n",
      "<15:23:45> F1.4 AVG:   OOF(r2=0.89984)  |  Time Elapsed: 0.04815 s\n",
      "<15:23:45> Repetition 1 AVG:   OOF(r2=0.86691)  |  Time Elapsed: 0.24459 s\n",
      "<15:23:45> \n",
      "<15:23:45> FINAL:    OOF(r2=0.88135)  |  Time Elapsed: 0.49638 s\n",
      "<15:23:45> \n",
      "<15:23:45> Saving results for Experiment: '3b699656-e442-4f21-afa8-80099e81427f'\n"
     ]
    }
   ],
   "source": [
    "from lightgbm import LGBMRegressor\n",
    "\n",
    "experiment = CVExperiment(\n",
    "    model_initializer=LGBMRegressor,\n",
    "    model_init_params=dict(\n",
    "        boosting_type=\"gbdt\", num_leaves=31, min_child_samples=5, subsample=0.5,\n",
    "    ),\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. Hyperparameter Optimization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validated Environment with key: \"2YztSDxi5paDQaugUYdWoxknmDUq4I-UBXQHCzXUIyg=\"\n",
      "\u001b[31mSaved Result Files\u001b[0m\n",
      "\u001b[31m_______________________________________________________________________________________\u001b[0m\n",
      " Step |       ID |   Time |      Value |   boosting_type |   num_leaves |   subsample | \n",
      "Experiments matching cross-experiment key/algorithm: 1\n",
      "Experiments fitting in the given space: 1\n",
      "Experiments matching current guidelines: 1\n",
      "    0 | 3b699656 | 00m00s | \u001b[35m   0.88135\u001b[0m | \u001b[32m           gbdt\u001b[0m | \u001b[32m          31\u001b[0m | \u001b[32m     0.5000\u001b[0m | \n",
      "\u001b[31mHyperparameter Optimization\u001b[0m\n",
      "\u001b[31m_______________________________________________________________________________________\u001b[0m\n",
      " Step |       ID |   Time |      Value |   boosting_type |   num_leaves |   subsample | \n",
      "    1 | 0a38ba6b | 00m00s |    0.82988 |            dart |           15 |      0.4684 | \n",
      "    2 | ebf7bde0 | 00m00s | \u001b[35m   0.88600\u001b[0m | \u001b[32m           gbdt\u001b[0m | \u001b[32m          29\u001b[0m | \u001b[32m     0.5947\u001b[0m | \n",
      "    3 | 2a145f7e | 00m00s |    0.83068 |            dart |           13 |      0.4824 | \n",
      "    4 | 270572c5 | 00m00s | \u001b[35m   0.88790\u001b[0m | \u001b[32m           gbdt\u001b[0m | \u001b[32m          21\u001b[0m | \u001b[32m     0.5386\u001b[0m | \n",
      "    5 | 37c275bc | 00m00s |    0.88441 |            gbdt |           13 |      0.4253 | \n",
      "    6 | 7104a56b | 00m00s |    0.83298 |            dart |           30 |      0.5206 | \n",
      "    7 | c0365235 | 00m00s |    0.82365 |            dart |           15 |      0.3583 | \n",
      "    8 | 23342f24 | 00m00s |    0.83682 |            dart |           40 |      0.6065 | \n",
      "    9 | 67c8c280 | 00m00s |    0.83294 |            dart |           30 |      0.6893 | \n",
      "   10 | b90fe642 | 00m00s | \u001b[35m   0.89244\u001b[0m | \u001b[32m           gbdt\u001b[0m | \u001b[32m          21\u001b[0m | \u001b[32m     0.6843\u001b[0m | \n",
      "   11 | 1cb47bb8 | 00m00s |    0.88725 |            gbdt |           18 |      0.4795 | \n",
      "   12 | cd96cd13 | 00m00s |    0.88843 |            gbdt |           31 |      0.5336 | \n",
      "Optimization loop completed in 0:00:05.904900\n",
      "Best score was 0.8924413185008423 from Experiment \"b90fe642-323c-46e6-8c0b-b932da3adfd1\"\n"
     ]
    }
   ],
   "source": [
    "from hyperparameter_hunter import ExtraTreesOptPro, Real, Integer, Categorical\n",
    "\n",
    "optimizer = ExtraTreesOptPro(iterations=12, random_state=1337)\n",
    "\n",
    "optimizer.forge_experiment(\n",
    "    model_initializer=LGBMRegressor,\n",
    "    model_init_params=dict(\n",
    "        boosting_type=Categorical([\"gbdt\", \"dart\"]),\n",
    "        num_leaves=Integer(10, 40),\n",
    "        max_depth=-1,\n",
    "        min_child_samples=5,\n",
    "        subsample=Real(0.3, 0.7),\n",
    "    ),\n",
    ")\n",
    "\n",
    "optimizer.go()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Notice, `optimizer` recognizes our earlier `experiment`'s hyperparameters fit inside the search space/guidelines set for `optimizer`.\n",
    "\n",
    "Then, when optimization is started, it automatically learns from `experiment`'s results - without any extra work for us!"
   ]
  }
 ],
 "metadata": {
  "celltoolbar": "Raw Cell Format",
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  },
  "pycharm": {
   "stem_cell": {
    "cell_type": "raw",
    "source": [],
    "metadata": {
     "collapsed": false
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}